{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Launcher notebook\n",
    "\n",
    "This notebook shows how to launch the flights_model.ipynb notebook either in Deep Learning VM or on Kubeflow pipelines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# change these to try this notebook out\n",
    "BUCKET = 'cloud-training-demos-ml'\n",
    "PROJECT = 'cloud-training-demos'\n",
    "REGION = 'us-central1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['BUCKET'] = BUCKET\n",
    "os.environ['PROJECT'] = PROJECT\n",
    "os.environ['REGION'] = REGION"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Launch \"locally\" using Python\n",
    "\n",
    "Make sure the notebook runs as intended when executed through papermill."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Install papermill package in the current Jupyter kernel\n",
    "import sys\n",
    "!{sys.executable} -m pip install papermill"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_flights_model(input_notebook: str, \n",
    "                      output_notebook: str, \n",
    "                      develop_mode: bool, \n",
    "                      bucket : str, \n",
    "                      project: str):\n",
    "  # execute notebook using papermill\n",
    "  import papermill as pm\n",
    "  pm.execute_notebook(\n",
    "    input_notebook,\n",
    "    output_notebook,\n",
    "    parameters = {'BUCKET': bucket, 'PROJECT': project, 'DEVELOP_MODE': develop_mode}\n",
    "  )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "rm -rf kfp_output\n",
    "mkdir kfp_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Input Notebook:  flights_model.ipynb\n",
      "Output Notebook: kfp_output/flights_model.ipynb\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5071812b82114d3b87e6024367fdbd43",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=30), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "run_flights_model('flights_model.ipynb',\n",
    "                  os.path.join('kfp_output', 'flights_model.ipynb'),\n",
    "                  develop_mode=True, bucket=BUCKET, project=PROJECT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "flights_model.ipynb\r\n"
     ]
    }
   ],
   "source": [
    "!ls kfp_output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Submit to Deep Learning VM to execute\n",
    "\n",
    "This will launch a beefy virtual machine on the cloud and execute the notebook.\n",
    "Change the params.yaml below to be your PROJECT and BUCKET"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting params.yaml\n"
     ]
    }
   ],
   "source": [
    "%%writefile params.yaml\n",
    "---\n",
    "BUCKET: cloud-training-demos-ml\n",
    "PROJECT: cloud-training-demos\n",
    "DEVELOP_MODE: False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Removing gs://cloud-training-demos-ml/flights/notebook/flights_model.ipynb#1543619551322929...\n",
      "Removing gs://cloud-training-demos-ml/flights/notebook/flights_small.ipynb#1543619551422089...\n",
      "Removing gs://cloud-training-demos-ml/flights/notebook/launcher.ipynb#1543619551539081...\n",
      "Removing gs://cloud-training-demos-ml/flights/notebook/params.yaml#1543619551633910...\n",
      "/ [1/4 objects]  25% Done                                                       \r",
      "/ [2/4 objects]  50% Done                                                       \r",
      "/ [3/4 objects]  75% Done                                                       \r",
      "/ [4/4 objects] 100% Done                                                       \r\n",
      "Operation completed over 4 objects.                                              \n",
      "Copying file://flights_model.ipynb [Content-Type=application/octet-stream]...\n",
      "/ [0 files][    0.0 B/ 27.9 KiB]                                                \r",
      "/ [1 files][ 27.9 KiB/ 27.9 KiB]                                                \r",
      "Copying file://flights_small.ipynb [Content-Type=application/octet-stream]...\n",
      "/ [1 files][ 27.9 KiB/ 35.6 KiB]                                                \r",
      "/ [2 files][ 35.6 KiB/ 35.6 KiB]                                                \r",
      "Copying file://launcher.ipynb [Content-Type=application/octet-stream]...\n",
      "/ [2 files][ 35.6 KiB/ 52.1 KiB]                                                \r",
      "/ [3 files][ 52.1 KiB/ 52.1 KiB]                                                \r",
      "Copying file://params.yaml [Content-Type=application/octet-stream]...\n",
      "/ [3 files][ 52.1 KiB/ 52.2 KiB]                                                \r",
      "/ [4 files][ 52.2 KiB/ 52.2 KiB]                                                \r\n",
      "Operation completed over 4 objects/52.2 KiB.                                     \n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "gcloud storage rm --recursive gs://$BUCKET/flights/notebook\n",
    "gcloud storage cp *.ipynb params.yaml gs://$BUCKET/flights/notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NAME              ZONE        MACHINE_TYPE   PREEMPTIBLE  INTERNAL_IP  EXTERNAL_IP     STATUS\n",
      "notebookexecutor  us-west1-b  n1-standard-4               10.138.0.2   35.247.113.103  RUNNING\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "+ GCS_FOLDER=gs://cloud-training-demos-ml/flights/notebook\n",
      "+ export IMAGE_FAMILY=tf-latest-cu100\n",
      "+ IMAGE_FAMILY=tf-latest-cu100\n",
      "+ export ZONE=us-west1-b\n",
      "+ ZONE=us-west1-b\n",
      "+ export INSTANCE_NAME=notebookexecutor\n",
      "+ INSTANCE_NAME=notebookexecutor\n",
      "+ export INSTANCE_TYPE=n1-standard-4\n",
      "+ INSTANCE_TYPE=n1-standard-4\n",
      "+ export INPUT_NOTEBOOK=gs://cloud-training-demos-ml/flights/notebook/flights_model.ipynb\n",
      "+ INPUT_NOTEBOOK=gs://cloud-training-demos-ml/flights/notebook/flights_model.ipynb\n",
      "+ export OUTPUT_NOTEBOOK_DIR=gs://cloud-training-demos-ml/flights/notebook\n",
      "+ OUTPUT_NOTEBOOK_DIR=gs://cloud-training-demos-ml/flights/notebook\n",
      "+ export PARAMS=gs://cloud-training-demos-ml/flights/notebook/params.yaml\n",
      "+ PARAMS=gs://cloud-training-demos-ml/flights/notebook/params.yaml\n",
      "+ export LAUNCHER_SCRIPT=https://raw.githubusercontent.com/b0noI/gcp-notebook-executor/master/notebook_executor.sh\n",
      "+ LAUNCHER_SCRIPT=https://raw.githubusercontent.com/b0noI/gcp-notebook-executor/master/notebook_executor.sh\n",
      "+ gcloud compute instances create notebookexecutor --zone=us-west1-b --image-family=tf-latest-cu100 --image-project=deeplearning-platform-release --maintenance-policy=TERMINATE --accelerator=type=nvidia-tesla-p100,count=1 --machine-type=n1-standard-4 --boot-disk-size=200GB --scopes=https://www.googleapis.com/auth/cloud-platform --metadata=parameters_file=gs://cloud-training-demos-ml/flights/notebook/params.yaml,input_notebook=gs://cloud-training-demos-ml/flights/notebook/flights_model.ipynb,output_notebook=gs://cloud-training-demos-ml/flights/notebook,startup-script-url=https://raw.githubusercontent.com/b0noI/gcp-notebook-executor/master/notebook_executor.sh\n",
      "Created [https://www.googleapis.com/compute/v1/projects/cloud-training-demos/zones/us-west1-b/instances/notebookexecutor].\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "\n",
    "set -x\n",
    "\n",
    "GCS_FOLDER=gs://$BUCKET/flights/notebook\n",
    "\n",
    "export IMAGE_FAMILY=\"tf-latest-cu100\" # or put any required\n",
    "export ZONE=\"us-west1-b\"\n",
    "export INSTANCE_NAME=\"notebookexecutor\"\n",
    "export INSTANCE_TYPE=\"n1-standard-4\"\n",
    "#export INPUT_NOTEBOOK=\"$GCS_FOLDER/flights_small.ipynb\"\n",
    "export INPUT_NOTEBOOK=\"$GCS_FOLDER/flights_model.ipynb\"\n",
    "export OUTPUT_NOTEBOOK_DIR=$GCS_FOLDER\n",
    "export PARAMS=\"$GCS_FOLDER/params.yaml\"\n",
    "export LAUNCHER_SCRIPT=https://raw.githubusercontent.com/b0noI/gcp-notebook-executor/master/notebook_executor.sh\n",
    "\n",
    "gcloud compute instances create $INSTANCE_NAME \\\n",
    "        --zone=$ZONE \\\n",
    "        --image-family=$IMAGE_FAMILY \\\n",
    "        --image-project=deeplearning-platform-release \\\n",
    "        --maintenance-policy=TERMINATE \\\n",
    "        --accelerator='type=nvidia-tesla-p100,count=1' \\\n",
    "        --machine-type=$INSTANCE_TYPE \\\n",
    "        --boot-disk-size=200GB \\\n",
    "        --scopes=https://www.googleapis.com/auth/cloud-platform \\\n",
    "        --metadata=\"parameters_file=$PARAMS,input_notebook=$INPUT_NOTEBOOK,output_notebook=$OUTPUT_NOTEBOOK_DIR,startup-script-url=$LAUNCHER_SCRIPT\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gs://cloud-training-demos-ml/flights/notebook/flights_small.ipynb\n",
      "gs://cloud-training-demos-ml/flights/notebook/notebook.ipynb\n",
      "gs://cloud-training-demos-ml/flights/notebook/params.yaml\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "# Note: gcloud storage mv is not atomic for cross-provider moves.\n",
    "gcloud storage mv gs://$BUCKET/flights/notebook/notebook.ipynb gs://$BUCKET/flights/notebook/flights_model_dlvm.ipynb\n",
    "gcloud storage ls gs://$BUCKET/flights/notebook"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Submit to Kubeflow pipelines to run as a component\n",
    "\n",
    "The submitnotebook docker image has the latest version of tensorflow and papermill installed. It invokes papermill on the supplied notebook."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "!{sys.executable} -m pip install https://storage.googleapis.com/ml-pipeline/release/0.1.3-rc.2/kfp.tar.gz --upgrade"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create the pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "flights_pipeline.tar.gz\n"
     ]
    }
   ],
   "source": [
    "import kfp.components as comp\n",
    "import kfp.dsl as dsl\n",
    "\n",
    "# a single-op pipeline that runs the flights pipeline on the pod\n",
    "@dsl.pipeline(\n",
    "   name='FlightsPipeline',\n",
    "   description='Trains, deploys flights model'\n",
    ")\n",
    "def flights_pipeline(\n",
    "   inputnb=dsl.PipelineParam('inputnb'),\n",
    "   outputnb=dsl.PipelineParam('outputnb'),\n",
    "   params=dsl.PipelineParam('params')\n",
    "):\n",
    "    notebookop = dsl.ContainerOp(\n",
    "      name='flightsmodel',\n",
    "      # image needs to be a compile-time string\n",
    "      image='gcr.io/cloud-training-demos/submitnotebook:latest',\n",
    "      arguments=[\n",
    "        inputnb,\n",
    "        outputnb,\n",
    "        params\n",
    "      ]\n",
    "    )\n",
    "\n",
    "# compile the pipeline\n",
    "pipeline_func = flights_pipeline\n",
    "pipeline_filename = pipeline_func.__name__ + '.tar.gz'\n",
    "import kfp.compiler as compiler\n",
    "compiler.Compiler().compile(pipeline_func, pipeline_filename)\n",
    "print(pipeline_filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "flights_pipeline.tar.gz\r\n"
     ]
    }
   ],
   "source": [
    "!ls *.tar.gz"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run the pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting params.yaml\n"
     ]
    }
   ],
   "source": [
    "%%writefile params.yaml\n",
    "---\n",
    "BUCKET: cloud-training-demos-ml\n",
    "PROJECT: cloud-training-demos\n",
    "DEVELOP_MODE: False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "gcloud storage rm --recursive gs://$BUCKET/flights/notebook\n",
    "gcloud storage cp *.ipynb params.yaml gs://$BUCKET/flights/notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gs://cloud-training-demos-ml/flights/notebook/flights_model.ipynb\r\n",
      "gs://cloud-training-demos-ml/flights/notebook/flights_small.ipynb\r\n",
      "gs://cloud-training-demos-ml/flights/notebook/launcher.ipynb\r\n",
      "gs://cloud-training-demos-ml/flights/notebook/params.yaml\r\n"
     ]
    }
   ],
   "source": [
    "!gcloud storage ls gs://$BUCKET/flights/notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Run link <a href=\"/pipeline/#/runs/details/34e935d6-f4f5-11e8-b1d6-42010a8000b8\" target=\"_blank\" >here</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Specify pipeline argument values\n",
    "GCSDIR='gs://{}/flights/notebook'.format(BUCKET)\n",
    "arguments = {\n",
    "    'inputnb': '{}/flights_model.ipynb'.format(GCSDIR),\n",
    "    'outputnb': '{}/flights_model_kfp.ipynb'.format(GCSDIR),\n",
    "    'params': '{}/params.yaml'.format(GCSDIR),\n",
    "}\n",
    "\n",
    "#Get or create an experiment and submit a pipeline run\n",
    "import kfp\n",
    "client = kfp.Client()\n",
    "list_experiments_response = client.list_experiments()\n",
    "experiments = list_experiments_response.experiments\n",
    "if not experiments:\n",
    "    #The user does not have any experiments available. Creating a new one\n",
    "    experiment = client.create_experiment('Flight pipeline experiment')\n",
    "else:\n",
    "    experiment = experiments[-1] #Using the last experiment\n",
    "\n",
    "#Submit a pipeline run\n",
    "from datetime import datetime\n",
    "run_name = 'Flight pipeline {}'.format(datetime.now().strftime(\"%Y%m%d %H%M%S\"))\n",
    "run_result = client.run_pipeline(experiment.id, run_name, pipeline_filename, params=arguments)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gs://cloud-training-demos-ml/flights/notebook/flights_model.ipynb\r\n",
      "gs://cloud-training-demos-ml/flights/notebook/flights_model_kfp.ipynb\r\n",
      "gs://cloud-training-demos-ml/flights/notebook/flights_small.ipynb\r\n",
      "gs://cloud-training-demos-ml/flights/notebook/launcher.ipynb\r\n",
      "gs://cloud-training-demos-ml/flights/notebook/notebook.ipynb\r\n",
      "gs://cloud-training-demos-ml/flights/notebook/params.yaml\r\n"
     ]
    }
   ],
   "source": [
    "!gcloud storage ls gs://$BUCKET/flights/notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
